/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */

package Builder;

import DAO.InformationFilter.PageDAO;
import DAO.WSPageDAO;
import Data.ListInputProcessor;
import DataImporter.DataImporter;
import Info.PageInfo;
import Manager.ExtractionManager;
import MySqlAccess.Article;
import MySqlAccess.ArticleInfo;
import Option.ModelOption;
import Processing.ExtractProcessor;
import java.util.ArrayList;
import java.util.List;
import Option.ModelManagerOption;
import Processing.Output.IOutput;
import Processing.Output.SimpleTable;
import Processing.Processor;
import java.sql.Timestamp;
import java.util.Calendar;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Set;
import util.Keyphrase;

/**
 *
 * @author tran tuan tai
 */
public class PageBuilder
{
    static String LF = System.getProperty("line.separator");

    private String ParseID(String content)
    {
        int iBegin = content.indexOf("<id>");
        int iEnd = content.indexOf("</id>");

        if (iBegin != -1 && iEnd != -1)
        {
            return content.substring(iBegin + 4, iEnd);
        }

        return "";
    }

    private void GetKeyphrase()
    {
        PageDAO pageDAO = new PageDAO();
        int begin = 0, end = 200;
        List<PageInfo> lstPage = pageDAO.GetPages_BOWNotExist(begin, end);

        ModelManagerOption option = new ModelManagerOption();
        option.ExtractDemo();
        option.SetPathModel("e:\\School\\2010\\Luan.Van\\Source\\ViAKI\\ViAKI\\model.dat");

        ExtractionManager exManager = new ExtractionManager();        
        ExtractProcessor extractor = (ExtractProcessor)exManager.InitExtractor(option);;
        extractor.SetNumberKey(10);

        while(!lstPage.isEmpty())
        {
            ListInputProcessor input = new ListInputProcessor();
            List<String> lstContent = new ArrayList<String>();
            for (PageInfo page : lstPage)
            {
                lstContent.add(page.GetContent());
            }
            input.Initialize(lstContent);
            IOutput output = new SimpleTable();
            extractor.SetOutput(output);
            extractor.SetInputProcessor(input);
            extractor.SetUseEvaluator(false);
            extractor.Process();

            HashMap<String, List<Keyphrase>> lstKey = output.GetAllDocKey();
            Set<String> lstDocID = lstKey.keySet();
            Iterator<String> iterID = lstDocID.iterator();
            while(iterID.hasNext())
            {
                String id = iterID.next();
                List<Keyphrase> keys = lstKey.get(id);
                StringBuilder bow = new StringBuilder();
                for (Keyphrase key : keys)
                {
                    bow.append(key.key).append('\t').append(key.tfidf).append('\t');
                }

                pageDAO.UpdateBOW(Integer.parseInt(id), bow.toString().trim());
            }

            System.out.println(lstKey.size());

            begin += 200;
            end += 200;

            lstPage = pageDAO.GetPages_BOWNotExist(begin, end);
        }
    }


    private void ImportData(Timestamp begin, Timestamp end)
    {
        Article article = new Article();
        List<ArticleInfo> allArt = article.GetByDate(begin, end);

        DataImporter importer = new DataImporter();
        importer.ConfigProxy("172.29.1.19", "8080");

        PageDAO pageDAO = new PageDAO();
        pageDAO.DoEmpty();

        WSPageDAO wsPageDAO = new WSPageDAO();

        for (ArticleInfo art : allArt)
        {
            String str = wsPageDAO.GetPageContent(art.id);
            if (str != null && !pageDAO.IsExist(art.id))
            {
                str = importer.Parse(str, art.url);
            }             

            if (str != null)
            {
                String myLink = ParseID(str);
                StringBuilder content = new StringBuilder();
                content.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
                content.append(LF);
                if (str.trim().isEmpty()) continue;
                content.append(str.replaceFirst("<id>\\S+</id>", "<id>" + String.valueOf(art.id) + "</id>" + LF + "<DOC>")).append("</DOC>");
                
                int id = pageDAO.Insert(art.id, myLink, content.toString(), art.post_date);
                System.out.println(id);
            }
        }
    }

    public void GetConcept()
    {}

    public void Clustering()
    {}

    public void Build()
    {
        //ImportData(new Timestamp(111, 5, 1, 0, 0, 0, 0), new Timestamp(Calendar.getInstance().getTime().getTime()));
        GetKeyphrase();
        //GetConcept();
        //Clustering();
    }


    public static void main(String[] agrvs)
    {
        PageBuilder builder = new PageBuilder();
        builder.Build();

//        PageDAO pageDAO = new PageDAO();
//        List<PageInfo> lstPage = pageDAO.GetAll();
//        for (PageInfo info : lstPage)
//        {
//            System.out.println(info.GetContent());
//        }

    }
}
